<html>
  <head>
   <meta charset="utf-8">
   <meta http-equiv="X-UA-Compatible" content="IE=edge">
   <meta name="viewport" content="width=device-width, initial-scale=1">
   <meta name="theme-color" content="#008000"/>
   <title>hands_ml</title>
   <link rel="manifest" href="hands_ml_manifest.json">
   <script>
    if('serviceWorker' in navigator) {
      navigator.serviceWorker.register('/hands_ml_sw.js')
        .then(function() {
          console.log('Service Worker Registered');
        });
    }
   </script>
   <style>
     body {
       margin: 0;
     }
   </style>
  </head>
  <body>
  <h1>hands_ml.html example</h1> 
  <script src="three.js"></script>
  <script>
    let container, scene, camera, session, controllerMeshes, handMeshes;
    // let handTracker;
    
    const boneVsh = `
      uniform vec3 offsetPosition;
      uniform vec4 offsetQuaternion;
      vec3 applyVectorQuaternion(vec3 v, vec4 q) {
        // return vec + 2.0 * cross( cross( vec, quat.xyz ) + quat.w * vec, quat.xyz );
        return v + 2.0 * cross(q.xyz, cross(q.xyz, v) + q.w * v);
      }
      void main() {
        gl_Position = projectionMatrix * modelViewMatrix * vec4(applyVectorQuaternion(position, offsetQuaternion) + offsetPosition, 1.);
      }
    `;
    const boneFsh = `
      void main() {
        gl_FragColor = vec4(1.0, 0.0, 0.0, 1.0);
      }
    `;

    function init() {
      container = document.createElement('div');
      document.body.appendChild(container);

      scene = new THREE.Scene();
      scene.matrixAutoUpdate = false;
      // scene.background = new THREE.Color(0x3B3961);

      camera = new THREE.PerspectiveCamera(60, window.innerWidth / window.innerHeight, 0.1, 1000);
      // camera.position.set(0, 1, 0);
      scene.add(camera);

      const ambientLight = new THREE.AmbientLight(0x808080);
      scene.add(ambientLight);

      const directionalLight = new THREE.DirectionalLight(0xFFFFFF, 1);
      directionalLight.position.set(1, 1, 1);
      scene.add(directionalLight);

      const localVector = new THREE.Vector3();
      const localVector2 = new THREE.Vector3();
      const localQuaternion = new THREE.Quaternion();
      const localMatrix = new THREE.Matrix4();
      const pointerGeometry = (() => {
        const geometries = [
          new THREE.CylinderBufferGeometry(0.001, 0.001, 1, 32, 1)
            .applyMatrix(localMatrix.makeTranslation(0, 1/2, 0))
            .applyMatrix(localMatrix.makeRotationFromQuaternion(
              localQuaternion.setFromUnitVectors(
                localVector.set(0, 1, 0),
                localVector2.set(0, 0, -1)
              )
            )),
          new THREE.BoxBufferGeometry(0.001, 0.1, 0.001)
            .applyMatrix(localMatrix.makeTranslation(0, 0.1/2, 0)),
        ];

        const positions = (() => {
          let numPositions = 0;
          for (let i = 0; i < geometries.length; i++) {
            numPositions += geometries[i].attributes.position.array.length;
          }

          const positions = new Float32Array(numPositions);
          let positionIndex = 0;
          for (let i = 0; i < geometries.length; i++) {
            const geometry = geometries[i];
            positions.set(geometry.attributes.position.array, positionIndex);
            positionIndex += geometry.attributes.position.array.length;
          }
          return positions;
        })();
        const normals = (() => {
          let numNormals = 0;
          for (let i = 0; i < geometries.length; i++) {
            numNormals += geometries[i].attributes.normal.array.length;
          }

          const normals = new Float32Array(numNormals);
          let normalIndex = 0;
          for (let i = 0; i < geometries.length; i++) {
            const geometry = geometries[i];
            normals.set(geometry.attributes.normal.array, normalIndex);
            normalIndex += geometry.attributes.normal.array.length;
          }
          return normals;
        })();
        const indices = (() => {
          let numIndices = 0;
          for (let i = 0; i < geometries.length; i++) {
            numIndices += geometries[i].index.array.length;
          }

          const indices = new Uint16Array(numIndices);
          let indexIndex = 0;
          let positionIndex = 0;
          for (let i = 0; i < geometries.length; i++) {
            const geometry = geometries[i];
            indices.set(geometry.index.array, indexIndex);

            const positionOffset = positionIndex / 3;
            for (let j = 0; j < geometry.index.array.length; j++) {
              indices[indexIndex + j] += positionOffset;
            }

            indexIndex += geometry.index.array.length;
            positionIndex += geometry.attributes.position.array.length;
          }
          return indices;
        })();

        const geometry = new THREE.BufferGeometry();
        geometry.addAttribute('position', new THREE.BufferAttribute(positions, 3));
        geometry.addAttribute('normal', new THREE.BufferAttribute(normals, 3));
        geometry.setIndex(new THREE.BufferAttribute(indices, 1));
        return geometry;
      })();
      const boneGeometry = new THREE.BoxBufferGeometry(0.01, 0.01, 0.01);
      const handMaterial =  new THREE.MeshPhongMaterial({
        color: 0xFF0000,
      });
      const boneMaterial = new THREE.ShaderMaterial({
        uniforms: {
          offsetPosition: {
            type: 'v3',
            value: new THREE.Vector3(),
          },
          offsetQuaternion: {
            type: 'v4',
            value: new THREE.Vector4(),
          },
        },
        vertexShader: boneVsh,
        fragmentShader: boneFsh,
      });
      const controllerMaterial = new THREE.MeshBasicMaterial({
        color: 0x66bb6a,
      });
      const controllerGaugeBackMaterial = new THREE.MeshBasicMaterial({
        color: 0x666666,
      });
      const controllerGaugeFrontMaterials = {
        normal: new THREE.MeshBasicMaterial({
          color: 0x01579b,
        }),
        touched: new THREE.MeshBasicMaterial({
          color: 0x03a9f4,
        }),
        pressed: new THREE.MeshBasicMaterial({
          color: 0xffffff,
        }),
      };
      const _makeControllerMesh = i => {
        const geometry = new THREE.BoxBufferGeometry(0.05, 0.1, 0.01)
          /* .applyMatrix(new THREE.Matrix4().makeRotationX(-Math.PI / 2))
          .applyMatrix(new THREE.Matrix4().makeTranslation(0, 0, -0.5)); */
        const material = controllerMaterial;

        const mesh = new THREE.Mesh(geometry, material);
        // mesh.position.y = -1;
        // mesh.updateMatrixWorld();
        // mesh.visible = true;
        mesh.frustumCulled = false;
        mesh.matrixAutoUpdate = false;
        
        const _makeFingerMesh = () => {
          const gaugeBackMesh = (() => {
            const geometry = new THREE.BoxBufferGeometry(0.01, 0.1, 0.01)
              .applyMatrix(new THREE.Matrix4().makeTranslation(0, 0, -0.01/2));
            const material = controllerGaugeBackMaterial;
            const mesh = new THREE.Mesh(geometry, material);
            // mesh.position.y = -0.1/2;
            mesh.frustumCulled = false;
            // mesh.matrixAutoUpdate = false;
            return mesh;
          })();
          const gaugeFrontMesh = (() => {
            const geometry = new THREE.BoxBufferGeometry(0.01, 0.1, 0.01)
              .applyMatrix(new THREE.Matrix4().makeTranslation(0, 0.1/2, 0.01/2));
            const material = controllerGaugeFrontMaterials.normal;
            const mesh = new THREE.Mesh(geometry, material);
            mesh.position.y = -0.1/2;
            mesh.frustumCulled = false;
            mesh.matrixAutoUpdate = false;
            return mesh;
          })();
          gaugeBackMesh.add(gaugeFrontMesh);
          gaugeBackMesh.setValue = (value, touched, pressed) => {
            if (value > 0) {
              gaugeFrontMesh.scale.y = value;
              gaugeFrontMesh.updateMatrix();
              gaugeFrontMesh.updateMatrixWorld();
              gaugeFrontMesh.visible = true;

              if (pressed) {
                gaugeFrontMesh.material = controllerGaugeFrontMaterials.pressed;
              } else if (touched) {
                gaugeFrontMesh.material = controllerGaugeFrontMaterials.touched;
              } else {
                gaugeFrontMesh.material = controllerGaugeFrontMaterials.normal;
              }
            } else {
              gaugeFrontMesh.visible = false;
            }
          };
          return gaugeBackMesh;
        };
        const fingerMeshes = Array(5);
        for (let j = 0; j < 5; j++) {
          const fingerMesh = _makeFingerMesh();
          fingerMesh.position.y = 0.1;
          fingerMesh.position.x = (-0.2 + j*0.1) * (i === 0 ? 1 : -1);
          // fingerMesh.visible = false;
          mesh.add(fingerMesh);
          fingerMeshes[j] = fingerMesh;
        }

        const _makeBoneMesh = () => {
          const geometry = new THREE.BoxBufferGeometry(0.01, 0.01, 0.01);
          // geometry.addAttribute('offsetPosition', new THREE.BufferAttribute(new Float32Array(3), 3));
          // geometry.addAttribute('offsetQuaternion', new THREE.BufferAttribute(new Float32Array(4), 4));
          const material = boneMaterial;
          const mesh = new THREE.Mesh(geometry, material);
          mesh.visible = false;
          mesh.frustumCulled = false;
          mesh.matrixAutoUpdate = false;
          mesh.onBeforeRender = () => {
            material.uniforms.offsetPosition.value.fromArray(mesh.offsetPosition);
            // material.uniforms.offsetPosition.value.set(j*0.1, i*0.1, 0);
            material.uniforms.offsetQuaternion.value.fromArray(mesh.offsetQuaternion);
          };
          mesh.offsetPosition = new Float32Array(3);
          mesh.offsetQuaternion = new Float32Array(4);
          return mesh;
        };
        const boneMeshes = Array(31);
        for (let j = 0; j < 31; j++) {
          const boneMesh = _makeBoneMesh();
          mesh.add(boneMesh);
          boneMeshes[j] = boneMesh;
        }

        mesh.update = frame => {
          if (session && frame) {
            const trackedPointerInputSources = session.getInputSources().filter(inputSource => inputSource.targetRayMode === 'tracked-pointer');
            const inputSource = trackedPointerInputSources[i];

            if (inputSource && inputSource.connected) {
              const pose = frame.getPose(inputSource.targetRaySpace);
              mesh.matrix.fromArray(pose.transform.inverse.matrix);
              mesh.matrixWorld.copy(mesh.matrix);
              // mesh.matrix.decompose(mesh.position, mesh.quaternion, mesh.scale);
              // mesh.updateMatrixWorld();
              // mesh.matrix.decompose(mesh.position, mesh.quaternion, mesh.scale);
            
              const gamepad = navigator.getGamepads()[i];
              const {buttons, bones} = gamepad;
              for (let j = 0; j < 5; j++) {
                const button = buttons[5+j];
                const fingerMesh = fingerMeshes[j];
                fingerMesh.updateMatrixWorld();
                fingerMesh.setValue(button.value, button.touched, button.pressed);
              }

              for (let j = 0; j < 31; j++) {
                const boneMesh = boneMeshes[j];
                const boneStartIndex = j*(3+4);
                if (!isNaN(bones[boneStartIndex])) {
                  // boneMesh.matrix.fromArray(bones, boneStartIndex);
                  boneMesh.matrixWorld.copy(mesh.matrixWorld);
                  // boneMesh.updateMatrixWorld();
                  boneMesh.offsetPosition.set(bones.slice(boneStartIndex, boneStartIndex + 3));
                  boneMesh.offsetQuaternion.set(bones.slice(boneStartIndex + 3, boneStartIndex + (3+4)));
                  boneMesh.visible = true;
                } else {
                  boneMesh.visible = false;
                }
              }

              mesh.visible = true;
            } else {
              mesh.visible = false;
            }
          }
        };
        
        return mesh;
      };
      controllerMeshes = [
        _makeControllerMesh(0),
        _makeControllerMesh(1),
      ];
      for (let i = 0; i < controllerMeshes.length; i++) {
        scene.add(controllerMeshes[i]);
      }

      const _makeHandMesh = i => {
        const geometry = new THREE.BufferGeometry();

        const positions = new Float32Array(boneGeometry.attributes.position.array.length * 2 * 6 * 4 + pointerGeometry.attributes.position.array.length * 2);
        const positionAttribute = new THREE.BufferAttribute(positions, 3);
        geometry.addAttribute('position', positionAttribute);

        const normals = new Float32Array(boneGeometry.attributes.normal.array.length * 2 * 6 * 4 + pointerGeometry.attributes.normal.array.length * 2);
        const normalAttribute = new THREE.BufferAttribute(normals, 3);
        geometry.addAttribute('normal', normalAttribute);

        const indices = new Uint16Array(boneGeometry.index.array.length * 2 * 6 * 4 + pointerGeometry.index.array.length * 2);
        const indexAttribute = new THREE.BufferAttribute(indices, 1);
        geometry.setIndex(indexAttribute);

        const mesh = new THREE.Mesh(geometry, handMaterial);
        mesh.update = frame => {
          if (session && frame) {
            const handInputSources = session.getInputSources().filter(inputSource => inputSource.targetRayMode === 'hand');
            const inputSource = handInputSources[i];

            if (inputSource && inputSource.connected) {
              let positionIndex = 0;
              let normalIndex = 0;
              let indexIndex = 0;

              const _shiftIndex = array => {
                for (let l = 0; l < array.length; l++) {
                  array[l] += positionIndex / 3;
                }
              };

              const pose = frame.getPose(inputSource.targetRaySpace);
              const {wrist, fingers} = inputSource;
              const allFingers = [wrist].concat(fingers);
              for (let j = 0; j < allFingers.length; j++) {
                const bones = allFingers[j];
                for (let k = 0; k < bones.length; k++) {
                  const positionFloat32Array = bones[k];
                  if (positionFloat32Array && isFinite(positionFloat32Array[0])) {
                    const newGeometry = boneGeometry.clone()
                      .applyMatrix(
                        localMatrix.makeTranslation(positionFloat32Array[0], positionFloat32Array[1], positionFloat32Array[2])
                      );

                    _shiftIndex(newGeometry.index.array);

                    positions.set(newGeometry.attributes.position.array, positionIndex);
                    positionIndex += newGeometry.attributes.position.array.length;

                    normals.set(newGeometry.attributes.normal.array, normalIndex);
                    normalIndex += newGeometry.attributes.normal.array.length;

                    indices.set(newGeometry.index.array, indexIndex);
                    indexIndex += newGeometry.index.array.length;
                  }
                }
              }

              /* if (pointer) {
                const newGeometry = pointerGeometry.clone()
                  .applyMatrix(
                    localMatrix
                      .compose(
                        localVector.fromArray(pointer.position),
                        localQuaternion.fromArray(pointer.rotation),
                        localVector2.set(1, 1, 1),
                      )
                  );

                _shiftIndex(newGeometry.index.array);

                positions.set(newGeometry.attributes.position.array, positionIndex);
                positionIndex += newGeometry.attributes.position.array.length;

                normals.set(newGeometry.attributes.normal.array, normalIndex);
                normalIndex += newGeometry.attributes.normal.array.length;

                indices.set(newGeometry.index.array, indexIndex);
                indexIndex += newGeometry.index.array.length;
              }

              if (grip) {
                const newGeometry = pointerGeometry.clone()
                  .applyMatrix(
                    localMatrix
                      .compose(
                        localVector.fromArray(grip.position),
                        localQuaternion.fromArray(grip.rotation),
                        localVector2.set(1, 1, 1),
                      )
                  );

                _shiftIndex(newGeometry.index.array);

                positions.set(newGeometry.attributes.position.array, positionIndex);
                positionIndex += newGeometry.attributes.position.array.length;

                normals.set(newGeometry.attributes.normal.array, normalIndex);
                normalIndex += newGeometry.attributes.normal.array.length;

                indices.set(newGeometry.index.array, indexIndex);
                indexIndex += newGeometry.index.array.length;
              } */

              positionAttribute.needsUpdate = true;
              indexAttribute.needsUpdate = true;
              geometry.setDrawRange(0, indexIndex);

              mesh.matrix.fromArray(pose.transform.inverse.matrix);
              mesh.matrix.decompose(mesh.position, mesh.quaternion, mesh.scale);
              mesh.updateMatrixWorld(true);
              
              mesh.visible = true;
            } else {
              mesh.visible = false;
            }
          } else {
            mesh.visible = false;
          }
        };
        mesh.visible = false;
        mesh.frustumCulled = false;
        return mesh;
      };
      handMeshes = [
        _makeHandMesh(0),
        _makeHandMesh(1),
      ];
      for (let i = 0; i < handMeshes.length; i++) {
        scene.add(handMeshes[i]);
      }
      /* const _onHands = hands => {
        handMesh.update(hands);

        for (let i = 0; i < hands.length; i++) {
          const hand = hands[i];
          const handIndex = hand.hand === 'left' ? 0 : 1;

          if (lastPincheds[handIndex]) {
            const handSpec = hand.pointer || hand.grip || null;

            if (handSpec) {
              const pinchedObject = pinchedObjects[handIndex];
              const pinchedObjectStart = pinchedObjectStarts[handIndex];

              pinchedObject.position.fromArray(handSpec.position);
              pinchedObject.quaternion
                .copy(pinchedObjectStart.rotation)
                .inverse()
                .premultiply(localQuaternion.fromArray(handSpec.rotation));
              pinchedObject.matrix.compose(pinchedObject.position, pinchedObject.quaternion, pinchedObject.scale)
              pinchedObject.updateMatrixWorld(true);
            }
          }
        }
      }; */
      const objectGeometry = new THREE.BoxBufferGeometry(0.1, 0.2, 0.001);
      const objectMaterial = new THREE.MeshPhongMaterial({
        color: 0x0000FF,
      });
      const lastPincheds = [false, false];
      const _makePinchedObject = () => {
        const mesh = new THREE.Mesh(objectGeometry, objectMaterial);
        mesh.frustumCulled = false;
        mesh.matrixAutoUpdate = false;
        mesh.visible = false;
        return mesh;
      };
      const pinchedObjects = [
        _makePinchedObject(),
        _makePinchedObject(),
      ];
      const pinchedObjectStarts = [
        {position: new THREE.Vector3(), rotation: new THREE.Quaternion()},
        {position: new THREE.Vector3(), rotation: new THREE.Quaternion()},
      ];
      const _onGesture = e => {
        const {hand, gesture, lastGesture, position, rotation} = e;
        console.log('got gesture', {hand, gesture, lastGesture});

        const pinched = gesture === 'pinch';
        const handIndex = hand === 'left' ? 0 : 1;
        const lastPinched = lastPincheds[handIndex];

        if (pinched && !lastPinched) {
          const pinchedObject = pinchedObjects[handIndex];
          pinchedObject.position.fromArray(position);
          pinchedObject.quaternion.fromArray(rotation);
          pinchedObject.matrix.compose(pinchedObject.position, pinchedObject.quaternion, pinchedObject.scale);
          pinchedObject.updateMatrixWorld(true);
          pinchedObject.visible = true;
          scene.add(pinchedObject);

          const pinchedObjectStart = pinchedObjectStarts[handIndex];
          pinchedObjectStart.position.fromArray(position);
          pinchedObjectStart.rotation.fromArray(rotation);
        }

        lastPincheds[handIndex] = pinched;
      };

      /* handTracker.onhands = _onHands;
      handTracker.ongesture = _onGesture; */
      // handMesh.visible = true;

      renderer = new THREE.WebGLRenderer({
        antialias: true,
        alpha: true,
      });
      renderer.setPixelRatio(window.devicePixelRatio);
      renderer.setSize(window.innerWidth, window.innerHeight);

      // window.browser.magicleap.RequestDepthPopulation(true);
      // renderer.autoClear = false;

      container.appendChild(renderer.domElement);

      renderer.setAnimationLoop(animate);
    }

    function animate(time, frame) {
      for (let i = 0; i < controllerMeshes.length; i++) {
        controllerMeshes[i].update(frame);
      }
      for (let i = 0; i < handMeshes.length; i++) {
        handMeshes[i].update(frame);
      }

      renderer.render(scene, renderer.vr.enabled ? renderer.vr.getCamera(camera) : camera);
    }

    init();

    (async () => {
      console.log('request session');
      session = await navigator.xr.requestSession({
        exclusive: true,
        extensions: {
          handTracking: true,
        },
      }).catch(err => Promise.resolve(null));

      if (session) {
        // console.log('request first frame');
        session.requestAnimationFrame((timestamp, frame) => {
          renderer.vr.setSession(session, {
            frameOfReferenceType: 'stage',
          });

          const {views} = frame.getViewerPose();
          const viewport = session.baseLayer.getViewport(views[0]);
          const width = viewport.width;
          const height = viewport.height;

          renderer.setSize(width * 2, height);

          renderer.setAnimationLoop(null);

          renderer.vr.enabled = true;
          renderer.vr.setAnimationLoop(animate);

          console.log('running!');
        });
      } else {
        console.log('no xr devices');
      }
    })();
  </script>
  </body>
</html>
